{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2f27b8db",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install twitter-api-client==0.9.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7885eecd",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from twitter.scraper import Scraper\n",
    "from twitter.util import *\n",
    "import pandas as pd\n",
    "\n",
    "scraper = Scraper(session=init_session())\n",
    "\n",
    "# example tweet\n",
    "tweet = scraper.tweets_details([1476988122986647553], limit=500, pbar=False)\n",
    "\n",
    "# unnest items and filter deleted tweets\n",
    "items = [y for x in find_key(tweet, 'items') for y in x if not find_key(y, 'tombstone')]\n",
    "\n",
    "# index into relevant data points\n",
    "tweet_results = [x.get('result') for x in find_key(items, 'tweet_results')]\n",
    "\n",
    "df = (\n",
    "    pd.json_normalize(tweet_results)\n",
    "    # remove duplicate replies if needed\n",
    "    .drop_duplicates('rest_id')\n",
    "    # clean up column names for illustrative purposes\n",
    "    .assign(date=lambda x: pd.to_datetime(x['legacy.created_at']).dt.strftime(\"%Y-%m-%d %H:%M:%S\"))\n",
    "    .assign(root_tweet=lambda x: x['legacy.conversation_id_str'])\n",
    "    .assign(text=lambda x: x['legacy.full_text'])\n",
    "    .assign(tweet=lambda x: x['rest_id'])\n",
    "    .assign(username=lambda x: x['core.user_results.result.legacy.screen_name'])\n",
    "    # sort by newest replies to root_tweet\n",
    "    .sort_values('date', ascending=False)\n",
    "    .reset_index(drop=True)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f5ddad65",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>username</th>\n",
       "      <th>tweet</th>\n",
       "      <th>text</th>\n",
       "      <th>root_tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2023-05-25 19:52:44</td>\n",
       "      <td>dtweedys</td>\n",
       "      <td>1661822628393254913</td>\n",
       "      <td>@MKBHD 🤺🔸💜 #SubTweets ftw 🔸🌑🕊️</td>\n",
       "      <td>1476988122986647553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2023-05-07 14:30:30</td>\n",
       "      <td>BallSchlonger</td>\n",
       "      <td>1655218553858015233</td>\n",
       "      <td>@jacknft_4 @MKBHD sorry i cant</td>\n",
       "      <td>1476988122986647553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2023-05-03 21:13:33</td>\n",
       "      <td>jacknft_4</td>\n",
       "      <td>1653870434075410436</td>\n",
       "      <td>@MKBHD Great! Dm me please</td>\n",
       "      <td>1476988122986647553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2023-05-02 17:38:40</td>\n",
       "      <td>tanmayshah28</td>\n",
       "      <td>1653453969476456450</td>\n",
       "      <td>@MKBHD Thank you! Lets too!</td>\n",
       "      <td>1476988122986647553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2023-04-02 14:08:55</td>\n",
       "      <td>iStevenPlays</td>\n",
       "      <td>1642529545415712768</td>\n",
       "      <td>@MKBHD I got you.\\n\\n‘🔥__🔥’</td>\n",
       "      <td>1476988122986647553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>2021-12-31 18:46:59</td>\n",
       "      <td>sbeams</td>\n",
       "      <td>1476988269187448832</td>\n",
       "      <td>@MKBHD https://t.co/1Fu5cKmPTn</td>\n",
       "      <td>1476988122986647553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>2021-12-31 18:46:48</td>\n",
       "      <td>woworeoo</td>\n",
       "      <td>1476988222609793033</td>\n",
       "      <td>@MKBHD Yeah Yeah no more tweets from you</td>\n",
       "      <td>1476988122986647553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201</th>\n",
       "      <td>2021-12-31 18:46:46</td>\n",
       "      <td>disisjorj</td>\n",
       "      <td>1476988217576640517</td>\n",
       "      <td>@MKBHD @vassizzle</td>\n",
       "      <td>1476988122986647553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>2021-12-31 18:46:46</td>\n",
       "      <td>TechWizYT</td>\n",
       "      <td>1476988215043276802</td>\n",
       "      <td>@MKBHD Great advice!</td>\n",
       "      <td>1476988122986647553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203</th>\n",
       "      <td>2021-12-31 18:46:41</td>\n",
       "      <td>0x_flea</td>\n",
       "      <td>1476988197146185730</td>\n",
       "      <td>@MKBHD ❤️🤘🏼</td>\n",
       "      <td>1476988122986647553</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>204 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    date       username                tweet   \n",
       "0    2023-05-25 19:52:44       dtweedys  1661822628393254913  \\\n",
       "1    2023-05-07 14:30:30  BallSchlonger  1655218553858015233   \n",
       "2    2023-05-03 21:13:33      jacknft_4  1653870434075410436   \n",
       "3    2023-05-02 17:38:40   tanmayshah28  1653453969476456450   \n",
       "4    2023-04-02 14:08:55   iStevenPlays  1642529545415712768   \n",
       "..                   ...            ...                  ...   \n",
       "199  2021-12-31 18:46:59         sbeams  1476988269187448832   \n",
       "200  2021-12-31 18:46:48       woworeoo  1476988222609793033   \n",
       "201  2021-12-31 18:46:46      disisjorj  1476988217576640517   \n",
       "202  2021-12-31 18:46:46      TechWizYT  1476988215043276802   \n",
       "203  2021-12-31 18:46:41        0x_flea  1476988197146185730   \n",
       "\n",
       "                                         text           root_tweet  \n",
       "0              @MKBHD 🤺🔸💜 #SubTweets ftw 🔸🌑🕊️  1476988122986647553  \n",
       "1              @jacknft_4 @MKBHD sorry i cant  1476988122986647553  \n",
       "2                  @MKBHD Great! Dm me please  1476988122986647553  \n",
       "3                 @MKBHD Thank you! Lets too!  1476988122986647553  \n",
       "4                 @MKBHD I got you.\\n\\n‘🔥__🔥’  1476988122986647553  \n",
       "..                                        ...                  ...  \n",
       "199            @MKBHD https://t.co/1Fu5cKmPTn  1476988122986647553  \n",
       "200  @MKBHD Yeah Yeah no more tweets from you  1476988122986647553  \n",
       "201                         @MKBHD @vassizzle  1476988122986647553  \n",
       "202                      @MKBHD Great advice!  1476988122986647553  \n",
       "203                               @MKBHD ❤️🤘🏼  1476988122986647553  \n",
       "\n",
       "[204 rows x 5 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[[\n",
    "    'date',\n",
    "    'username',\n",
    "    'tweet',\n",
    "    'text',\n",
    "    'root_tweet',\n",
    "]]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
